Utilities.py

### Copyright (C) 2020-2023  Alessio Gianelle, INFN Padova
###
### This program is free software: you can redistribute it and/or modify
### it under the terms of the GNU General Public License as published by
### the Free Software Foundation, either version 3 of the License, or
### later version.
###
### This program is distributed in the hope that it will be useful,
### but WITHOUT ANY WARRANTY; without even the implied warranty of
### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
### GNU General Public License for more details.
###
### You should found a copy of the GNU General Public License
### at this path: https://www.gnu.org/licenses/ .

# -*- coding: utf-8 -*-
import pandas as pd
import numpy as np
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import itertools
from datetime import datetime
from sklearn.utils import shuffle
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import auc, roc_curve, confusion_matrix


# to change backend on the fly
# plt.switch_backend('tkAgg')

def pmsg(lvl, thr, msg):
  ''' 
  print msg if lvl <= thr       
  lvl 0 -> MESSAGE
  lvl 1 -> VERBOSE
  lvl 2 -> DEBUG
  '''
  if lvl <= thr:
    label="MESSAGE"
    if lvl == 1:
      label = "VERBOSE"
    if lvl == 2:
      label = "DEBUG"
    print("%s %s --- %s\n"%(datetime.now().strftime('%D-%H:%M:%S'), label, msg))


def readPop(filepath, label=0, sheet=None, selectfeatures="", newfeatures=""):
  if sheet is None:
    raw = pd.read_excel(filepath) 
  else:
    raw = pd.read_excel(filepath, sheet_name=sheet)
  if selectfeatures:
    pop = raw[selectfeatures]
  else:
    pop = raw
  if newfeatures:
    pop.columns = newfeatures
  pop['label'] = label
  return pop


def prepareData(popA, popB, applyPCA=False, thrPCA=0.95, scaleData=True, shuffleData=True, equal=False):
  # Concatenate the two population dataframe
  # If required apply PCA with threshold = "thrPCA"
  # Data are normalized with StandardScaler and shuffled (if required)
  # If equal use the same number of data for both populations (the min)
  # Return two dataframes: Data and Labels
  if equal:
    num = min(popA.shape[0], popB.shape[0])
    popB = popB.sample(n=num, random_state=37)   
    popA = popA.sample(n=num, random_state=73)   
  rawdata = pd.concat([popA, popB], axis=0)
  rawdata.reset_index(inplace=True, drop=True)
  data  = rawdata.drop(columns=['label'])
  label = rawdata.label
  if shuffleData:
    data, label = shuffle(data, label)
  if scaleData:
    scaledData = StandardScaler().fit_transform(data)
    data = scaledData
  if applyPCA:
    pca = PCA(thrPCA)
    trainData = pca.fit_transform(data)
    data = trainData
  return data, label

def showplot(filename=None):
  '''
  If filename is defined save plot else display it in a non block way
  '''
  if filename is None:
    plt.show(block=False);
  else:
    plt.savefig("%s"%filename)
    plt.close()

def confmatrix(cm, classes, filename=None,
                normalize=False,
                title='Confusion matrix',
                cmap=plt.cm.RdYlGn):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    if normalize:
        cm = cm.astype('float') / (100*cm.sum(axis=1)[:, np.newaxis])
    plt.rcParams["axes.grid"] = False
    plt.figure()
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes)
    plt.yticks(tick_marks, classes)
    fmt = '.2%' if normalize else 'd'
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),
          horizontalalignment="center",
          color="white" if cm[i, j] > thresh else "black")
    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')
    showplot(filename)


def history(hist, filename):
  '''
  Plot Accuracy and loss plots from fit history data
  '''
  plt.figure()
  plt.plot(hist.history['accuracy'])
  plt.plot(hist.history['val_accuracy'])
  plt.title('model accuracy')
  plt.ylabel('accuracy')
  plt.xlabel('epoch')
  plt.legend(['train', 'eval'], loc='upper left')
  fname = None if not filename else "%s_Acc%s"%(filename[:-4], filename[-4:])
  showplot(fname)
  # summarize history for loss
  plt.figure()
  plt.plot(hist.history['loss'])
  plt.plot(hist.history['val_loss'])
  plt.title('model loss')
  plt.ylabel('loss')
  plt.xlabel('epoch')
  plt.legend(['train', 'eval'], loc='upper left')
  fname = None if not filename else "%s_Loss%s"%(filename[:-4], filename[-4:])
  showplot(fname)

def plotroc(fpr, tpr, auc, filename=None):
  ''' 
  Plot ROC AUC
  '''
  plt.figure()
  plt.plot(fpr, tpr, color='darkorange',
           lw=2, label='ROC curve (area = %0.3f)' % auc)
  plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')
  plt.xlim([0.0, 1.0])
  plt.ylim([0.0, 1.05])
  plt.xlabel('mistag (FPR)')
  plt.ylabel('efficiency (TPR)')
  plt.title('Receiver operating characteristic')
  plt.legend(loc="lower right")
  showplot(filename)



